#coding:utf-8
"""
*   TG_Coder : Dzlua
*   Email    : 505544956@qq.com
*   Time     : 2017/08/28
"""

from database import DataBase
from bs4 import BeautifulSoup
import re
import codecs

import string, sys, os

import ftplib

from progressbar import *

'''
    1-> get path
    2-> copy to 4T
    3-> unpack
    4-> del binary
    5-> pack
    6-> upload
'''
class Upload(DataBase):
    ftp = ftplib.FTP()
    ftp.set_pasv(False)
    ftp_root = 'data'
    def __init__(self, host, user, pwd, db):
        DataBase.__init__(self, host, user, pwd, db)
    def setpath(self, src1, src2, dst):
        self.__src1 = src1
        self.__src2 = src2
        self.__dst = dst
    def setftproot(self, root):
        Upload.ftp_root = root    
    def write(self, text):
        f = codecs.open('upload.log', 'a', 'utf-8')
        f.write(text)
        f.close()
    def setlogin(self, host, port, user, pwd):
        self._ftp_host = host
        self._ftp_port = port
        self._ftp_user = user
        self._ftp_pwd = pwd
    def login(self):
        self.ftp.connect(self._ftp_host, self._ftp_port)
        self.ftp.login(self._ftp_user, self._ftp_pwd)
        print(self.ftp.welcome)
    def start(self, bid, eid):
        self.read(self.__on_read, [bid, eid])
    def close(self):
        DataBase.close(self)
        self.ftp.close()
    def uploadfile(self, localfile, remotefile, callback, args=None):
        if os.path.isfile(localfile) == False:
            return False

        data = {
            'persize': 1024,
            'allsize': os.path.getsize(localfile),
            'localfile': localfile,
            'remotefile': remotefile,
            'times': 0,
            'size': 0,
            'finish': False
        }

        def funcallback(buf):
            data['times'] += 1
            data['buffer'] = buf
            data['size'] += len(buf)
            data['finish'] = (data['size'] >= data['allsize'])
            pbar.update(data['size'])
            callback(data, args)
        
        self.login()
        self.ftpmkdirs(remotefile, True)

        widgets = ['[Upload] uploading : ', Percentage(), ' ', Bar(marker=RotatingMarker('>-=')),
           ' ', ETA(), ' ', FileTransferSpeed()]
        pbar = ProgressBar(widgets=widgets, maxval=data['allsize']).start()

        file_handler = open(localfile, "rb", data['persize'])
        self.ftp.storbinary('STOR %s' % remotefile, file_handler, data['persize'], funcallback)
        file_handler.close()
        pbar.finish()
        return True
    def ftpmkdirs(self, path, bfile=False):
        def _mkdirs(dir):
            dirs = dir.split('/')
            if len(dirs)<=1:
                return
            cur = '.'
            for d in dirs:
                cur += '/' + d
                try:
                    self.ftp.mkd(cur)
                except:
                    pass
        #
        pt = path
        if bfile:
            pt = path[0 : path.rfind('/')]
        _mkdirs(pt)
    def write(self, text):
        f = codecs.open('update.log', 'a', 'utf-8')
        f.write(text)
        f.close()
    def _select(self, conn, sql):
        cursor = conn.cursor()
        cursor.execute(sql)
        result = cursor.fetchall()
        cursor.close()
        return result
    def _getCount(self, conn):
        sql = 'SELECT count(*) FROM librarys;'
        result = self._select(conn, sql)

        count = 0
        for r in result:
            count = r[0]
            break
        return count
    def _getPath(self, data):
        abs =  u'/' + data['path'] + u'/' + data['name']
        path1 = self.__src1 + abs
        path2 = self.__src2 + abs
        dst = self.__dst + abs
        return path1, path2, dst
    def _checkfile(self, file):
        if os.path.isfile(file):
            return os.path.getsize(file)
        return False
    def _checkdir(self, dir):
        if not os.path.isdir(dir):
            return False
        if os.listdir(dir):
            return self._getdirsize(dir)
        return False
    def _getdirsize(self, dir):
        size = 0l
        for (root,dirs,files) in os.walk(dir):
            for name in files:
                try:
                    size += os.path.getsize(os.path.join(root,name))
                except:
                    continue
        return size
    def istextfile(self, filename, blocksize = 512):
        if os.path.isfile(filename) == False:
            return False
        return self.istext(open(filename).read(blocksize))
    def istext(self, s):
        text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
        _null_trans = string.maketrans("", "")

        if "\0" in s:
            return False
    
        if not s:
            return False
        t = s.translate(_null_trans, text_characters)
        if float(len(t))/float(len(s)) > 0.30:
            return False
        return True
    def mkdirs(self, path, dir=False):
        pt = path
        try:
            if dir:
                os.makedirs(path)
                return
            pt = path[0 : path.rfind('/')]
            os.makedirs(pt)
        except:
            print('[Upload] mkdirs error,(exist?): %s' % pt)
    def deldirfile(self, path):
        try:
            cmd = 'rm -rf %s' % path
            #print '[Upload] deldirfile : %s' % cmd
            os.system(cmd)
        except:
            print('[Upload] deldirfile error,(?): %s' % path)
    def copy(self, src, dst):
        cmd = 'cp -rf %s %s' % (src, dst)
        print '[Upload] copy : %s' % cmd
        ret = ( os.system(cmd) == 0 )
        assert(ret)
        print '[Upload] copy : done!'
    def filedir(self, file, times=1):
        pos = file.rfind('/')
        dir = file[0 : pos]
        name = file[pos : len(file)]
        while times > 0:
            pos = name.rfind('.', 0, pos)
            times -= 1
        return dir, name[len('/') : pos], name[pos : len(name)]
    def unpack_zip(self, file):
        dir,name,type = self.filedir(file, 1)
        dst = dir + '/' + name
        
        #del dir
        self.deldirfile(dst)
        #create dir
        self.mkdirs(dst, True)
        #unzip
        cmd = 'unzip -q %s -d %s' % (file, dst)
        print '[Upload] unpack zip : %s' % cmd
        ret = ( os.system(cmd) == 0 )
        assert(ret)
        #del zip
        self.deldirfile(file)

        print '[Upload] unpack zip : done!'

        return dir,name,type
    def unpack_tar(self, file):
        dir,name,type = self.filedir(file, 2)
        dst = dir + '/' + name

        #del dir
        self.deldirfile(dst)
        #create dir
        self.mkdirs(dst, True)
        #untar
        cmd = 'tar -zxf %s -C %s' % (file, dst)
        print '[Upload] unpack tar : %s' % cmd
        ret = ( os.system(cmd) == 0 )
        assert(ret)
        #del tar
        self.deldirfile(file)

        print '[Upload] unpack tar : done!'

        return dir,name,type
    def pack_tar(self, dir, name):
        file = dir + '/' + name + '.tar.gz'
        cmd = 'tar -zcf %s -C %s %s' % (file, dir, name)
        print '[Upload] pack tar : %s' % cmd
        ret = ( os.system(cmd) == 0 )
        assert(ret)
        self.deldirfile(dir + '/' + name)
        print '[Upload] pack tar : done!'
        return file,name + '.tar.gz'
    def upload(self, file, name, tpath):
        remotepath = self.ftp_root + '/' + tpath + '/' + name
        print '[Upload] upload package : local: %s, remote: %s' % (file, remotepath)
        ret = self.uploadfile(file, remotepath, self.__on_upload)
        assert(ret)
        print '[Upload] upload package : done!'
    def __on_read(self, conn, args):
        bid = args[0]
        eid = args[1]

        count = self._getCount(conn)
        id = bid
        while id < count and id <= eid:
            print('------------------------------')
            print('[Upload] id:%s/%d[%s,%s], num:%s' % (id, count, bid, eid, id-bid+1) )
            sql = "SELECT id,name,local_path FROM librarys WHERE id=%s;" % id
            result = self._select(conn, sql)
            for r in result:
                info = {
                    'id': r[0],
                    'name': r[1],
                    'path': r[2]
                }
                try:
                    self._getPjo(info)
                except:
                    errmsg = '[Upload] ERROR id:%s,name:%s' % (id, info['name'])
                    self.write(errmsg + '\n')
                    print(errmsg)
                break
            id = id +1
    def __on_upload(self, data, args):
        pass
    def _checkdelfiles(self, path):
        def _delfile(file):
            bt = self.istextfile(file)
            if not bt:
                self.deldirfile(file)
        #
        for (root,dirs,files) in os.walk(path):
            for file in files:
                _delfile(root + '/' + file)
            for dir in dirs:
                self._checkdelfiles(root + dir)
    def _getPjo(self, data):
        def _file(src):
            zip = src + u'.zip'
            tar = src + u'.tar.gz'
            git = src
            return zip.encode('utf8'), tar.encode('utf8'), git.encode('utf8')
        #
        src1,src2,dst = self._getPath(data)
        dst = dst.encode('utf8')
        tpath = data['path'].encode('utf8')
        for src in [src1, src2]:
            zip,tar,git = _file(src)
            
            #
            res = self._checkfile(zip)
            if res:
                if self._zip(res, zip, dst + '.zip', tpath):
                    break
            #
            res = self._checkfile(tar)
            if res:
                if self._tar(res, tar, dst + '.tar.gz', tpath):
                    break
            #
            res = self._checkdir(git)
            if res:
                if self._git(res, git, dst, tpath):
                    break
    def _zip(self, res, src, dst, tpath):
        print '[Upload] zip: %s, src: %s, dst: %s' % (res, src, dst)
        self.deldirfile(dst)
        self.mkdirs(dst)
        self.copy(src, dst)
        dir,name,type = self.unpack_zip(dst)
        print('[Upload] checking binary : ...')
        self._checkdelfiles(dir + '/' + name)
        print('[Upload] checking binary : done')
        pkg,name = self.pack_tar(dir, name)
        self.upload(pkg, name, tpath)
        return True
    def _tar(self, res, src, dst, tpath):
        print '[Upload] tar: %s, src: %s, dst: %s' % (res, src, dst)
        self.deldirfile(dst)
        self.mkdirs(dst)
        self.copy(src, dst)
        dir,name,type = self.unpack_tar(dst)
        print('[Upload] checking binary : ...')
        self._checkdelfiles(dir + '/' + name)
        print('[Upload] checking binary : done')
        pkg,name = self.pack_tar(dir, name)
        self.upload(pkg, name, tpath)
        return True
    def _git(self, res, src, dst, tpath):
        print '[Upload] git: %s, src: %s, dst: %s' % (res, src, dst)
        self.deldirfile(dst)
        self.mkdirs(dst)
        self.copy(src, dst)
        dir,name,type = self.filedir(dst, 0)
        print('[Upload] checking binary : ...')
        self._checkdelfiles(dir + '/' + name)
        print('[Upload] checking binary : done')
        pkg,name = self.pack_tar(dir, name)
        self.upload(pkg, name, tpath)
        return True
#------------
def readdatas(file):
    datas = []
    f = open(file)
    
    pattern = re.compile(r'(\d)+')

    while True:
        line = f.readline()
        if not line:
            break
        match = pattern.search(line)
        if match:
            datas.append(match.group())
    f.close()
    return datas
#------------
upload = Upload('localhost', 'dzlua', 'dzlua', 'spider')
#upload.setlogin('120.27.13.210', '21', 'zh', '505544956')
upload.setlogin('120.27.12.202', '21', 'zh', '505544956')
'''
upload.setpath(u'/home/zhouhan/dzlua/tg-data',
            u'/home/zhouhan/dzlua/tg-data2',
            u'/media/zhouhan/5E8CA1E98CA1BC41/tg-data' )
upload.setftproot('data2')
'''
upload.setftproot('data2')
upload.setpath(u'/home/zhouhan/dzlua/tg-data',
            u'/home/zhouhan/dzlua/tg-data2',
            u'/home/zhouhan/dzlua/tg-data-back/data')
datas = readdatas('update.data')
for id in datas:
    iid = int(id)
    upload.start(iid, iid)
upload.close()
# for error
